#!/usr/bin/env python 
# -*- coding: utf-8 -*- 
'''
Created on 2016年5月16日
@author: zhaoxiang
'''

import sys,os
import json
import re
from urlparse import urlparse
import urllib
from UrlParser import UrlParser
import MySQLdb
import MySQLdb.cursors
import re
import pymongo

reload(sys)
sys.setdefaultencoding("utf8")


class DB:
    def __init__(self, host, username, pwd, db, port = 3306) :
        self.myHost = host
        self.myUsername = username
        self.myPassword = pwd
        self.myDatabase = db
        if host != "mysql2" : self.myPort = int(port)
        else : self.myPort = 3307
        self.connect()
        self.cursor = None

    def connect(self):
        self.conn = MySQLdb.connect(host=self.myHost, user=self.myUsername, passwd=self.myPassword, db=self.myDatabase, port = self.myPort, charset='utf8', cursorclass=MySQLdb.cursors.DictCursor)

    #处理分表，已被sql_util中horizonal_sql代替
    def preprocess_sql(self, sql, data = None) :
        tablename_match = re.search("from[ ]+(\S+)", sql)
        if not tablename_match : return [(sql, data)]
        table_name = tablename_match.group(1)
        if table_name in ["plan_oneday_h", "plan_onedayevent_h"] :
            if data : query_sql = sql % tuple(map(str, data))
            else : query_sql = sql
            plan_id_match = re.search("plan_id[ ]+=[ ]+(\d+)", query_sql)
            if plan_id_match :
                plan_id = plan_id_match.group(1)
                if table_name == "plan_oneday_h" : table_index = int(plan_id) % 20
                else : table_index = int(plan_id) % 50
                sql = sql.replace(table_name, "%s_%d" % (table_name, table_index))
                return [(sql, data)]
#                 res_sql_datas.append((sql, data))
            else :
                plan_ids_match = re.search(r"plan_id[ ]+in[ ]+\(([\d ,]*)\)", query_sql)
                if plan_ids_match :
                    res_sql_datas = []
                    tableindex_planids_dict = {}
                    plan_ids = plan_ids_match.group(1).split(",")
                    for plan_id in plan_ids :
                        if table_name == "plan_oneday_h" : table_index = int(plan_id) % 20
                        else : table_index = int(plan_id) % 50
                        if table_index not in tableindex_planids_dict : tableindex_planids_dict[table_index] = [plan_id]
                        else : tableindex_planids_dict[table_index].append(plan_id)
                    for tableindex, planids in tableindex_planids_dict.iteritems() :
                        new_sql = sql.replace(table_name, "%s_%d" % (table_name, tableindex)).replace(plan_ids_match.group(0), "plan_id in (%s)" % ",".join(planids))
                        res_sql_datas.append((new_sql, None))
                    if len(res_sql_datas) : return res_sql_datas
                    else : return [(sql, data)]
                else : return [(sql, data)]
        else : return [(sql, data)]

    #执行sql, 查询参数为data
    def execute(self, sql, data = None) :
        try :
            self.cursor = self.conn.cursor()
            if data :
                self.cursor.execute(sql, data)
                self.conn.commit()
            else :
                self.cursor.execute(sql)
                self.conn.commit()
        except (AttributeError, MySQLdb.OperationalError) :
            self.connect()
            self.cursor = self.conn.cursor()
            if data :
                self.cursor.execute(sql, data)
                self.conn.commit()
            else :
                self.cursor.execute(sql)
                self.conn.commit()
        return

    #执行多条语句, datalist
    def executemany(self, sql, datalist) :
        try :
            self.cursor = self.conn.cursor()
            self.cursor.executemany(sql, datalist)
            self.conn.commit()
        except (AttributeError, MySQLdb.OperationalError) :
            self.connect()
            self.cursor = self.conn.cursor()
            self.cursor.executemany(sql, datalist)
            self.conn.commit()

    #获取一条结果
    def fetchone(self) :
        if self.cursor :
            result = self.cursor.fetchone()
            self.cursor.close()
            self.cursor = None
            return result
        else : return None

    #获取所有结果
    def fetchall(self) :
        if self.cursor :
            res_set = self.cursor.fetchall()
            self.cursor.close()
            self.cursor = None
            return res_set
        else : return None

    #获取多条结果
    def fetchmany(self, rownum) :
        if self.cursor :
            res_set = self.cursor.fetchmany(rownum)
            if len(res_set) < rownum :
                self.cursor.close()
                self.cursor = None
            return res_set
        else : return None

    #关闭数据库连接
    def close(self):
        if self.cursor : self.cursor.close()
        if self.conn : self.conn.close()
        
class MongoClient(object) :
    def __init__(self, host, db, port = 27001) :
        self.host = host
        self.port = int(port)
        self.db = db

        self.conn = pymongo.Connection(self.host, self.port)
        self.database = self.conn[self.db]

    def execute(self, table, query = "", mode = "select") :
        if query == "" : return None
        try : 
            if mode == "selectone" : return self.database[table].find_one(query)
            if mode == "select" : return self.database[table].find(query)
            if mode == "insert" : return self.database[table].insert(query)
            if mode == "delete" : return self.database[table].remove(query)
            if mode == "update" : return self.database[table].update(query)
            if mode == "index" : return self.database[table].create_index(query)
        except Exception, e :
            print ">>> Mongodb Error [%s]: %s" % (mode,e)
            return None

    def close(self) :
        self.conn.disconnect()


#字符串解码
def strdecode(s):
    try :
        return s.decode('utf8')
    except UnicodeDecodeError :
        try :
            return s.decode('gbk')
        except UnicodeDecodeError :
            try :
                return s.decode('gb2312')
            except UnicodeDecodeError :
                try :
                    return s.decode('gb18030')
                except UnicodeDecodeError :
                    try :
                        return s.decode('gbk', 'ignore')
                    except UnicodeDecodeError :
                        try :
                            return s.decode('utf8', 'ignore')
                        except UnicodeDecodeError :
                            return s

    return s



#全站Url解析器
class UrlParser(object) :
    def __init__(self) :
        self.city_urlname = {}
        self.country_urlname = {}
        self.continent_urlname = {}
        self.hotel_urls = {}
        self.guide_urlname = {}

    #####################################################
    #功能:根据锦囊英文名，初始化锦囊url                 #
    #输出:字典 {锦囊url名:锦囊id}                       #
    #####################################################
    def init_guide_urlname(self) :
        guide_urlname = {}
        qyer_db = DB(conn_qyer["host"], conn_qyer["username"], conn_qyer["password"], conn_qyer["db"], conn_qyer["port"])
        qyer_db.execute("select id, enname from jn_info where status = 1")
        result_set = qyer_db.fetchall()
        qyer_db.close()
        for result in result_set :
            guide_id = result["id"]
            enname = result["enname"].lower()
            urlname = enname.replace("'", "").replace(" ", "-").replace("&", "and").replace(",", "-").replace(".", "")
            guide_urlname[urlname] = guide_id
        return guide_urlname

    #####################################################
    #功能:根据城市英文名，初始化城市url                 #
    #输出:字典 {城市url名:城市id}                       #
    #####################################################
    def init_city_urlname(self) :
        city_urlname = {}
        qyer_db = DB(conn_qyer["host"], conn_qyer["username"], conn_qyer["password"], conn_qyer["db"], conn_qyer["port"])
        qyer_db.execute("select id, cityname, cityname_en, cityname_url from city where status = 1")
        result_set = qyer_db.fetchall()
        qyer_db.close()
        for result in result_set :
            cityid = result["id"]
            enname = result["cityname_en"].lower().strip("\t ")
            cityname_url = result["cityname_url"]
            if cityname_url : enname = cityname_url.lower().strip("\t ")
            urlname = enname.replace("'", "").replace(" ", "-").replace("&", "and").replace(",", "-").replace(".", "")
            city_urlname[urlname] = cityid
        return city_urlname

    #####################################################
    #功能:根据国家英文名，初始化国家url                 #
    #输出:字典 {国家url名:国家id}                       #
    #####################################################
    def init_country_urlname(self) :
        country_urlname = {}
        qyer_db = DB(conn_qyer["host"], conn_qyer["username"], conn_qyer["password"], conn_qyer["db"], conn_qyer["port"])
        qyer_db.execute("select id, catename, catename_en from category where parentid in (select id from category where parentid = 1)")
        result_set = qyer_db.fetchall()
        for result in result_set :
            country_id = result["id"]
            enname = result["catename_en"].lower()
            urlname = enname.replace("'", "").replace(" ", "-").replace("&", "and").replace(",", "-").replace(".", "")
            country_urlname[urlname] = country_id
        return country_urlname

    #####################################################
    #功能:根据大洲英文名，初始化大洲url                 #
    #输出:字典 {大洲url名:大洲id}                       #
    #####################################################
    def init_continent_urlname(self) :
        continent_urlname = {}
        qyer_db = DB(conn_qyer["host"], conn_qyer["username"], conn_qyer["password"], conn_qyer["db"], conn_qyer["port"])
        qyer_db.execute("select id, catename_en from category where parentid = 1")
        result_set = qyer_db.fetchall()
        for result in result_set :
            continent_id = result["id"]
            enname = result["catename_en"].lower()
            urlname = enname.replace("'", "").replace(" ", "-").replace("&", "and").replace(",", "-").replace(".", "")
            continent_urlname[urlname] = continent_id
        return continent_urlname
        
    #####################################################
    #功能:根据酒店对应的booking链接，初始化酒店url      #
    #输出:字典{booking链接:酒店id}                      #
    #####################################################
    def init_hotel_url(self) :
        qyer_db = DB(conn_qyer["host"], conn_qyer["username"], conn_qyer["password"], conn_qyer["db"], conn_qyer["port"])
        qyer_db.execute("select id, booking_url from business_hotel where source = 1")
        result_set = qyer_db.fetchall()
        qyer_db.close()
        hotel_urls = {result["booking_url"] : result["id"] for result in result_set if result["booking_url"]}
        return hotel_urls

    ################################################################
    #功能:按产品线解析url                                          #
    #输入:url                                                      #
    #输出:字典 {”action_type":行为类型, "action_content":行为内容  #
    ################################################################
    def parse_url(self, url) :
        parsedUrl = urlparse(url)
        #web端锦囊
        if parsedUrl.netloc == "guide.qyer.com" : return self.parse_guide_url(parsedUrl.path, parsedUrl.query)
        #m端/app端锦囊
        if (parsedUrl.netloc == "m.qyer.com" or parsedUrl.netloc == "appview.qyer.com") and parsedUrl.path.startswith("/guide") : return self.parse_guide_url("/"+parsedUrl.path.split("/", 2)[-1], parsedUrl.query)
        #web端目的地
        if parsedUrl.netloc == "place.qyer.com" : return self.parse_place_url(parsedUrl.path, parsedUrl.query)
        #m端/app端目的地
        if (parsedUrl.netloc == "m.qyer.com" or parsedUrl.netloc == "appview.qyer.com") and parsedUrl.path.startswith("/place") : return self.parse_place_url("/"+parsedUrl.path.split("/", 2)[-1], parsedUrl.query)
        #web端问答
        if parsedUrl.netloc == "ask.qyer.com" : return self.parse_ask_url(parsedUrl.path, parsedUrl.query)
        #m端/app端问答
        if (parsedUrl.netloc == "m.qyer.com" or parsedUrl.netloc == "appview.qyer.com") and parsedUrl.path.startswith("/ask") : return self.parse_ask_url("/"+parsedUrl.path.split("/", 2)[-1], parsedUrl.query)
        #web端行程助手
        if parsedUrl.netloc == "plan.qyer.com" : return self.parse_plan_url(parsedUrl.path, parsedUrl.query)
        #m端/app端行程助手
        if (parsedUrl.netloc == "m.qyer.com" or parsedUrl.netloc == "appview.qyer.com") and parsedUrl.path.startswith("/plan") : return self.parse_plan_url("/"+parsedUrl.path.split("/", 2)[-1], parsedUrl.query)
        #web端论坛
        if parsedUrl.netloc == "bbs.qyer.com" : return self.parse_bbs_url(parsedUrl.path, parsedUrl.query)
        #m端/app端论坛
        if (parsedUrl.netloc == "m.qyer.com" or parsedUrl.netloc == "appview.qyer.com") and parsedUrl.path.startswith("/bbs") : return self.parse_bbs_url("/"+parsedUrl.path.split("/", 2)[-1], parsedUrl.query)
        if (parsedUrl.netloc == "m.qyer.com" or parsedUrl.netloc == "appview.qyer.com") and parsedUrl.path.startswith("/path") : return self.parse_bbs_url("/"+parsedUrl.path.split("/", 2)[-1], parsedUrl.query)
        #web端折扣
        if parsedUrl.netloc == "z.qyer.com" : return self.parse_z_url(parsedUrl.path, parsedUrl.query)
        #m端/app端折扣
        if (parsedUrl.netloc == "m.qyer.com" or parsedUrl.netloc == "appview.qyer.com") and parsedUrl.path.startswith("/z/") : return self.parse_z_url("/"+parsedUrl.path.split("/", 2)[-1], parsedUrl.query)
        #web端搜索
        if parsedUrl.netloc == "search.qyer.com" : return self.parse_search_url(parsedUrl.path, parsedUrl.query)
        #m端搜索
        if parsedUrl.netloc == "m.qyer.com" and (parsedUrl.path.startswith("/s.php") or parsedUrl.path.startswith("/search")) : return self.parse_search_url("/"+parsedUrl.path.split("/", 2)[-1], parsedUrl.query)
        #跳转booking
        if parsedUrl.path.startswith("/goto.php") and parsedUrl.query.find("booking.com") != -1 : return self.parse_goto_hotel_url(parsedUrl.query)
        return {"action_type":"view_others", "action_content":{}}

    ################################################################
    #功能:解析锦囊页                                               #
    #输入:path - string, 二级路径                                  #
    #     query - string, url中查询参数                            #
    #输出:字典 {”action_type":行为类型, "action_content":行为内容  #
    ################################################################
    def parse_guide_url(self, path, query) :
        if len(self.guide_urlname) == 0 : self.guide_urlname = self.init_guide_urlname()
        if not path or path == "/" : return {"action_type":"view_guide_index", "action_content":{}}
        words = path.split("/")
        if len(words) > 1 : return {"action_type":"view_guide_detail", "action_content":{"guide_name":words[1], "guide_id":self.guide_urlname.get(words[1],0)}}
        else : return {"action_type":"view_guide_others", "action_content":{}}

    ################################################################
    #功能:解析目的地页                                             #
    #输入:path - string, 二级路径                                  #
    #     query - string, url中查询参数                            #
    #输出:字典 {”action_type":行为类型, "action_content":行为内容  #
    ################################################################
    def parse_place_url(self, path, query) :
        if len(self.city_urlname) == 0 : self.city_urlname = self.init_city_urlname() 
        if len(self.country_urlname) == 0 : self.country_urlname = self.init_country_urlname()
        if len(self.continent_urlname) == 0 : self.continent_urlname = self.init_continent_urlname()
        words = path.split("/")
        if len(words) > 1 :
            if words[1] == "poi" :
                m = re.search(r"poi/(\d+)", path)
                if not m : return {"action_type":"view_poi_others", "action_content":{}}
                poi_id = int(m.group(1))
                if "photo"in path : return {"action_type": "view_poi_photo_detail", "action_content":{"poi_id":poi_id}}
                return {"action_type":"view_poi", "action_content":{"poi_id":poi_id}}
            elif words[1] in self.city_urlname :
                city_id = self.city_urlname[words[1]]
                if "mguide" in path : return {"action_type":"view_city_mguide_list", "action_content":{"city_id":city_id}}
                if "hotel" in path : return {"action_type":"view_city_hotel_list", "action_content":{"city_id":city_id}}
                if "photo" in path : return {"action_type":"view_city_photo_detail", "action_content":{"city_id":city_id}} 
                return {"action_type":"view_city", "action_content":{"city_id":city_id}}
            elif words[1] in self.country_urlname :
                country_id = self.country_urlname[words[1]]
                if "mguide" in path : return {"action_type":"view_country_mguide_list", "action_content":{"country_id":country_id}}
                if "hotel" in path : return {"action_type":"view_country_hotel_list", "action_content":{"country_id":country_id}}
                if "photo" in path : return {"action_type":"view_country_photo_detail", "action_content":{"country_id":country_id}}
                return {"action_type":"view_country", "action_content":{"country_id":country_id}}
            elif words[1] in self.continent_urlname :
                continent_id = self.continent_urlname[words[1]]
                if "mguide" in path : return {"action_type":"view_continent_mguide_list", "action_content":{"continent_id": continent_id}}
                if "hotel" in path : return {"action_type":"view_continent_hotel_list", "action_content":{"continent_id" : continent_id}}
                if "photo" in path : return {"action_type":"view_continent_hotel_list", "action_content":{"continent_id":continent_id}}
                return {"action_type":"view_continent", "action_content":{"continent_id":continent_id}}
            elif words[1] == "mguide" :
                m = re.search(r"(\d+)", path)
                if m :
                    mguide_id = int(m.group(1))
                    return {"action_type":"view_mguide_detail", "action_content":{"mguide_id":mguide_id}}
                else : return {"action_type":"view_mguide_others", "action_content":{}}
            elif words[1] == "" : return {"action_type":"view_place_index", "action_content":{}}
            else : return {"action_type":"view_place_others", "action_content":{}}
        else : return {"action_type":"view_place_index", "action_content":{}}

    ################################################################
    #功能:解析问答页                                               #
    #输入:path - string, 二级路径                                  #
    #     query - string, url中查询参数                            #
    #输出:字典 {”action_type":行为类型, "action_content":行为内容  #
    ################################################################
    def parse_ask_url(self, path, query) :
        m = re.search(r"question/(\d+)", path)
        if m :
            question_id = int(m.group(1))
            return {"action_type":"view_ask_detail", "action_content":{"qid":question_id}}
        elif path.startswith("/tag") : 
            temps = path.split("/")
            if len(temps) >= 3 : return {"action_type":"view_ask_tag", "action_content":{"tag":temps[2]}}
            else : return {"action_type":"view_ask_tag", "action_content":{"tag":""}}
        elif path.startswith("/search") :
            query_url = urllib.unquote(urllib.unquote(query))
            search_result = self.parse_search(query_url)
            return {"action_type":"search_ask", "action_content":search_result}
        else : return {"action_type":"view_ask_list", "action_content":{}}


    ################################################################
    #功能:解析行程助手页                                           #
    #输入:path - string, 二级路径                                  #
    #     query - string, url中查询参数                            #
    #输出:字典 {”action_type":行为类型, "action_content":行为内容  #
    ################################################################
    def parse_plan_url(self, path, query) :
        if "/create" in path : return {"action_type":"create_plan", "action_content":{}} 
        if "/smartcreate" in path : return {"action_type":"smartcreate_plan", "action_content":{}}
        if path.startswith("/search") :
            query_url = urllib.unquote(urllib.unquote(query)) 
            search_result = self.parse_search(query_url)
            temps = path.split("_")
            if len(temps) >= 4 : search_result["countryids"] = [int(temps[3])]
            return {"action_type":"search_plan", "action_content":search_result}
        if "/calendar" in path or "/cost" in path or "/message" in path or "/map" in path or "/traffic" in path or "/hotel" in path :
            m = re.search(r"/(\d+)/", path)
            if m  :
                plan_id = int(m.group(1))
                return {"action_type":"view_plan_detail", "action_content":{"plan_id":plan_id}}
            else : return {"action_type":"view_plan_detail", "action_content":{}}
        m = re.search(r"trip/(\d+)", path)
        if m : 
            plan_id = int(m.group(1))
            if "edit" in path : return {"action_type":"edit_plan", "action_content":{"plan_id":plan_id}}
            else : return {"action_type":"view_plan_detail", "action_content":{"plan_id":plan_id}} 
        else :
            m = re.search(r"tripapp/(\d+)", path)
            if m : 
                plan_id = int(m.group(1))
                return {"action_type":"view_plan_detail", "action_content":{"plan_id":plan_id}}
            else :
                m = re.search(r"poi_(\d+)", path)
                if m :
                    poi_id = int(m.group(1))
                    return {"action_type":"view_place_poi", "action_content":{"poi_id":poi_id}}
                else : return {"action_type":"view_plan_others", "action_content":{}}

    ################################################################
    #功能:解析论坛页                                               #
    #输入:path - string, 二级路径                                  #
    #     query - string, url中查询参数                            #
    #输出:字典 {”action_type":行为类型, "action_content":行为内容  #
    ################################################################
    def parse_bbs_url(self, path, query) :
        if "post.php" in path : return {"action_type":"post_thread", "action_content":{}}
        if "search.php" in path : 
            query_url = urllib.unquote(urllib.unquote(query))
            search_result = self.parse_search(query_url)
            return {"action_type":"search_bbs", "action_content":search_result}
        tid = 0
        m = re.search(r"thread-(\d+)-", path)
        if m : tid = int(m.group(1))
        else :
            m = re.search(r"tid=(\d+)", query)
            if m : tid = int(m.group(1))
        if tid : return {"action_type":"view_thread", "action_content":{"tid":tid}}
        fid = 0
        m = re.search(r"forum-(\d+)", path)
        if m : fid = int(m.group(1))
        if fid : return {"action_type":"view_bbs_forum", "action_content":{"fid":fid}}
        return {"action_type":"view_bbs_others", "action_content":{}} 

    ################################################################
    #功能:解析折扣页                                               #
    #输入:path - string, 二级路径                                  #
    #     query - string, url中查询参数                            #
    #输出:字典 {”action_type":行为类型, "action_content":行为内容  #
    ################################################################
    def parse_z_url(self, path, query) :
        query_url = urllib.unquote(urllib.unquote(query))
        product_type_ids = {"flights":1016, "hotels":1017, "free":1018, "freetour":1018, "cruise":1020, "car":1021, "safe":1049, "qz":1785, "local":2410, "ot":2161, "other":1050, "cityfun":2410}
        m = re.search(r"/deal/(\d+)", path)
        if m : 
            lm_id = int(m.group(1))
            return {"action_type":"view_lm_detail", "action_content":{"lm_id":lm_id}}
        elif path.startswith("/zt") :
            temps = path.split("/")
            if len(temps) >= 3 : return {"action_type":"view_lm_zt", "action_content":{"zt_name":temps[2]}}
            else : return {"action_type":"view_lm_zt", "action_content":{"zt_name":""}}
        elif path.startswith("/sup") :
            temps = path.split("/")
            if len(temps) >= 3 : 
                if temps[2].isdigit() : return {"action_type":"view_lm_sup", "action_content":{"sup_id":int(temps[2])}}
                elif temps[-1].isdigit() : return {"action_type":"view_lm_sup", "action_content":{"sup_id":int(temps[-1])}}
            else : return {"action_type":"view_lm_sup", "action_content":{"sup_id":0}}
        elif path.startswith("/orderform") : return {"action_type":"lm_orderfom"}
        else :
            m = re.search("action=orderform&lid=(\d+)", query) 
            if m : return {"action_type":"lm_orderform", "action_content":{"lm_id":int(m.group(1))}}
        if query.startswith("ptype") or query.startswith("dpt") or query.startswith("date") or query.startswith("continent") or query.startswith("cnty") or query.startswith("kw") or query.startswith("turn") :
            params = query_url.split("&")
            search_type = 0
            dept = 0
            travel_time = 0
            continent_id = 0
            country_id = 0
            city_id = 0
            kw = ""
            for param in params :
                try : k, v = param.split("=")
                except : continue
                if k == "ptype" : search_type = product_type_ids.get(v, 0)
                elif k == "dpt" : dept = int(v) if v.isdigit() else  0
                elif k == "date" : travel_time = int(v) if v.isdigit() else 0
                elif k == "continent" : continent_id = int(v) if v.isdigit() else 0
                elif k == "cnty" : country_id = int(v) if v.isdigit() else 0
                elif k == "country" : country_id = int(v) if v.isdigit() else 0
                elif k == "city" : city_id = int(v) if v.isdigit() else  0
                elif k == "kw" : kw = v
            return {"action_type":"view_lm_list", "action_content":{"product_type":search_type, "dpt":dept, "travel_time":travel_time, "continent_id":continent_id, "country_id":country_id, "city_id":city_id, "kw":kw}}
        else :
            m = re.search("(\d+)_(\d+)_(\d+)_(\d+)", path)
            if m : 
                if path.startswith("/free") or path.startswith("/qz") or path.startswith("/hotels") or path.startswith("/flights") or path.startswith("/other") or path.startswith("/cruise") or path.startswith("/car") or path.startswith("/safe") or path.startswith("/local") or path.startswith("/ot") or path.startswith("/cityfun"): 
                    #print path
                    name = path.split("/")[1]
                    if name in product_type_ids :
                        search_type = product_type_ids[name]
                        temps = path.split("/")[2:]
                    else :
                        name = name.split("_")[0]
                        if name in product_type_ids :
                            search_type = product_type_ids[name]
                            temps = ["all_" + path.split("_", 1)[1]]
                        else : return {"action_type":"view_lm_list", "action_content":{"product_type":0, "dpt":0, "travel_time":0, "continent_id":0, "country_id":country_id, "city_id":city_id, "kw":""}}
                    #print "xxx", search_type, temps
                else : 
                    if query.startswith("_channel=cityfun") : search_type = product_type_ids["local"]
                    else : search_type = 0
                    temps = path.split("/")[1:]
                dept = 0
                travel_time = 0
                continent_id = 0
                country_id = 0
                city_id = 0
                if len(temps) :
                    temp = temps[0].split("_")
                    if len(temp) == 4 :
                        try : dept = int(temp[0])
                        except : dept = 0
                        try : travel_time = int(temp[1])
                        except : travel_time = 0
                        try : continent_id = int(temp[2])
                        except : continent_id = 0
                        try : country_id = int(temp[3])
                        except : country_id = 0
                        if country_id == 1966 : country_id = 11; city_id = 50
                        elif country_id == 1967 : country_id = 11; city_id = 51
                        elif country_id == 1968 : country_id = 11; city_id = 11186
                    else :
                        try : dept = int(temp[1])
                        except : dept = 0
                        try : travel_time = int(temp[2])
                        except : travel_time = 0
                        #try : continent_id = temp[3]
                        #except : continent_id = 0
                        try : country_id = temp[3]
                        except : country_id = 0
                        try : city_id = temp[4] 
                        except : city_id = 0
                return {"action_type":"view_lm_list", "action_content":{"product_type":search_type, "dpt":dept, "travel_time": travel_time, "continent_id":continent_id, "country_id":country_id, "city_id":city_id, "kw":""}}
            elif path.startswith("/free") or path.startswith("/qz") or path.startswith("/hotels") or path.startswith("/flights") or path.startswith("/other") or path.startswith("/cruise") or path.startswith("/car") or path.startswith("/safe") or path.startswith("/local") or path.startswith("/ot") or path.startswith("/cityfun"): 
                search_type = product_type_ids[path.split("/")[1]]
                return {"action_type":"view_lm_list", "action_content":{"product_type":search_type, "dpt":0, "travel_time":0, "content_id":0, "country_id":0, "city_id":0, "kw":""}}
            m = re.search("aciton=detail&id=(\d+)", query)
            if m : return {"action_type":"view_lm_detail", "action_content":{"lm_id":int(m.group(1))}}
            else : return {"action_type":"view_lm_others", "action_content":{}}
                    
    ################################################################
    #功能:解析搜索页                                               #
    #输入:path - string, 二级路径                                  #
    #     query - string, url中查询参数                            #
    #输出:字典 {”action_type":行为类型, "action_content":行为内容  #
    ################################################################
    def parse_search_url(self, path, query) :
        if path.startswith("/index") : search_range = "all"
        elif path.startswith("/bbs") : search_range = "bbs"
        elif path.startswith("/plan") : search_range = "plan"
        elif path.startswith("/lastminute") : search_range = "lm"
        elif path.startswith("/ask") : search_range = "ask"
        elif path.startswith("/user") : search_range = "user"
        elif path.startswith("/hotel") : search_range = "hotel"
        else : search_range = "all"
        m = re.search(r"(\d+)", path)
        if m : return {"action_type":"search_%s" % search_range, "action_content":{"wd_code":int(m.group(1))}}
        query_url = urllib.unquote(urllib.unquote(query))
        search_result = self.parse_search(query_url)
        return {"action_type":"search_%s" % search_range, "action_content":search_result}

    ################################################################
    #功能:解析搜索页                                               #
    #输入:path - string, 二级路径                                  #
    #     query - string, url中查询参数                            #
    #输出:字典 {”action_type":行为类型, "action_content":行为内容  #
    ################################################################
    def parse_search(self, query) :
        search_result = {}
        m = re.search(ur"(wd|q|tags|keyword)=([^&]*)", query)
        if m : search_result["wd"] = strdecode(m.group(2))
        m = re.search(r"(cityid)=([\d,]+)", query)
        if m : 
            city_ids = map(int, m.group(2).split(","))
            search_result["cityids"] = city_ids
        return search_result
    
    ################################################################
    #功能:解析跳转酒店页                                           #
    #输入:path - string, 二级路径                                  #
    #     query - string, url中查询参数                            #
    #输出:字典 {”action_type":行为类型, "action_content":行为内容  #
    ################################################################
    def parse_goto_hotel_url(self, query) :
        if len(self.hotel_urls) == 0 : self.hotel_urls = self.init_hotel_url()
        query_url = urllib.unquote(urllib.unquote(query))
        m = re.search("hotelid=(\d+)", query_url)
        if m :
            hotel_id = int(m.group(1))
            return {"action_type":"goto_hotel", "action_content":{"hotel_id":hotel_id}}
        else :
            m = re.search("url=([^?]+)", query_url)
            if m :
                booking_url = m.group(1)
                hotel_id = self.hotel_urls.get(booking_url, 0)
                if hotel_id : return {"action_type":"goto_hotel", "action_content":{"hotel_id":hotel_id}}
        return {"action_type":"goto_hotel", "action_content":{}}

#数据库连接配置文件
conn_ra= {'host':'mysql-main', 'username':'root', 'password':'qyer_www2013_super', 'db':'product', "port":"3306"}
conn_qyer = {'host':'mysql2', 'username':'ra_house', 'password':'3JWOFlrah3VxraBeu', 'db':'qyer', 'port':'3307'}
conn_qyer_b = {'host':'mysql2', 'username':'ra_house', 'password':'3JWOFlrah3VxraBeu', 'db':'qyer', 'port':'3307'}
conn_plan = {'host':'mysql_plan', 'username':'ra_house', 'password':'3JWOFlrah3VxraBeu', 'db':'qyer_plan', 'port':'3308'}
conn_api = {'host':'mysql-main', 'username':'root', 'password':'qyer_www2013_super', 'db':'product'}
conn_main = {'host':'MainDB', 'username':'ra_db', 'password':'ra_db_2013china', 'db':'qyer', 'port':'3306'}
conn_plan_main = {'host':'MainDB_plan', 'username':'ra_db', 'password':'ra_db_2013china', 'db':'qyer_plan'}
conn_planagent = {'host':'127.0.0.1', 'username':'ra_house', 'password':'3JWOFlrah3VxraBeu', 'db':'qyer', 'port':'1234'}
conn_qyer_redis = {'host':'10.1.1.15'}
conn_ra_redis = {"host":"redis", "port":4670}
conn_218 = {'host':'10.1.1.218', 'username':'jiaqi', 'password':'qyer_123456', 'db':'interior'}
conn_219 = {'host':'10.1.1.219', 'username':'jiaqi', 'password':'585988Snowie', 'db':'ra'}
conn_booking = {'host':'10.1.1.218', 'username':'jiaqi', 'password':'qyer_123456', 'db':'interior'}
conn_hive = {"host":"hiveServer", "username":"jiaqi", "password":"", "db":"qyer"}
conn_mongo = {"host":"mongo-r", "username":"", "password":"", "db":"dc"}
db_name = {"mongo_ra":"ra"}
conn_mongo = {"host":"mongo-r", "username":"", "password":"", "db":"dc"}

mongo_db = MongoClient(conn_mongo["host"], conn_mongo["db"])
uid_doraid = {}
guid_doraid = {}
def extract_web_action(rowSet) :
#     sql = "select user_id, unique_id, ugc_type, ugc_content, path from user_tracks where type = 1 and platform in ('web', 'mobile') and logdate = %s and loghour = %s and path not like 'www.qyer.com/u/%%' and path not like 'plan.qyer.com/trip/%%/edit/%%' and unique_id not in ('07f77925-39c4-1f13-c803-1e98c2a8e06d','b7d0d478-f36b-359f-da47-95219b3fa8ed','c11d90af-4cdf-0bb3-0593-445e1a92492a','e118ff8f-525c-53cb-20cd-c6a594598ee3','8872c5d2-3c5b-2a18-d95a-4a7987a55231','53f72fa1-647e-863b-7838-7e2fdd9db3c1','caf60000-d989-24af-6f90-64a2fe71f30d','1b5e4005-8bf6-4d64-cd02-4b57c5b02251','77ca2ce4-cc3d-1e65-5614-f5270d57c8b3','2940daa2-e5d2-1626-ed15-d59de84fc010','2940daa2-e5d2-1626-ed15-d59de84fc010','66692910-2239-852e-7636-a1a3dbd0682d')" % (theday.strftime("%Y%m%d"),theday.strftime("%H"))
    urlparser = UrlParser()
    if len(rowSet) < 5 : 
        return
    dora_id = 0
    #根据uid或者guid解析dora_id
    uid = rowSet[0]
    if uid.isdigit() : 
        uid = int(uid)
    else : 
        uid = 0
    if uid :
        if uid in uid_doraid : 
            dora_id = uid_doraid[uid]
        else :
            uid_rowSet = mongo_db.execute("uid_doraid_1", {"_id":uid}, "selectone")
            if uid_rowSet : 
                dora_id = uid_rowSet["dora_id"]
                uid_doraid[uid] = dora_id
    guid = rowSet[1]
    if len(guid) == 36 :
        if dora_id == 0 :
            if guid in guid_doraid : 
                dora_id = guid_doraid[guid]
            else :
                guid_rowSet = mongo_db.execute("guid_doraid_1", {"_id":guid}, "selectone")
                if guid_rowSet : 
                    dora_id = guid_rowSet["dora_id"]
                    guid_doraid[guid] = dora_id
                    if uid : 
                        uid_doraid[uid] = dora_id
        else : guid_doraid[guid] = dora_id
    if dora_id <= 0 : 
        print '%s\t%s\t%s' % (uid, guid, rowSet[2])
    #解析ugc内容
    ugc_type = rowSet[2]
    ugc_content = rowSet[3]
    path = "http://" + rowSet[4]
    url = strdecode(urllib.unquote(urllib.unquote(path)))
    parsedUrl = urlparse(url)
    
    parsed_type=None
    parsed_content=None

    if ugc_type == "bbs_thread" or ugc_type == "mbbs_thread" :
        if ugc_content.isdigit() : 
            tid = int(ugc_content)
            parsed_type = "v_tbbs"
            parsed_content = tid
    elif ugc_type == "ask_question" or ugc_type == "mask_question" :
        if ugc_content.isdigit() :
            qid = int(ugc_content)
            parsed_type = "v_ask"
            parsed_content = qid
    elif ugc_type == "plan_trip" or ugc_type == "mplan_trip" :
        if ugc_content.isdigit() :
            plan_id = int(ugc_content)
            parsed_type = "v_plan"
            parsed_content = plan_id
    elif ugc_type == "plan_edit" or ugc_type == "mplan_edit" :
        if ugc_content.isdigit() :
            plan_id = int(ugc_content)
            parsed_type = "edit_plan"
            parsed_content = plan_id
    elif ugc_type == "place_country" or ugc_type == "mplace_country" :
        if ugc_content.isdigit() :
            country_id = int(ugc_content)
            parsed_type = "v_place"
            parsed_content = {"country":country_id}
    elif ugc_type == "place_city" or ugc_type == "mplace_city" :
        if ugc_content.isdigit() :
            city_id = int(ugc_content)
            parsed_type = "v_place"
            parsed_content = {"city":city_id}
    elif ugc_type == "place_poi" or ugc_type == "mplace_poi" :
        if ugc_content.isdigit() :
            poi_id = int(ugc_content)
            parsed_type = "v_place"
            parsed_content = {"poi":poi_id}
    elif ugc_type == "place_mguide" or ugc_type == "mplace_mguide" :
        if ugc_content.isdigit() :
            mguide_id = int(ugc_content)
            parsed_type = "v_mguide"
            parsed_content = mguide_id
    elif ugc_type == "guide_detail" or ugc_type == "mguide_detail" :
        if ugc_content.isdigit() :
            guide_id = int(ugc_content)
            parsed_type = "v_guide"
            parsed_content = guide_id
    elif ugc_type == "z_detail" or ugc_type == "mz_detail" :
        if ugc_content.isdigit() :
            lm_id = int(ugc_content)
            parsed_type = "v_lm"
            parsed_content = lm_id
    elif ugc_type == "hotel_city" :
        if ugc_content.isdigit() :
            city_id = int(ugc_content)
            parsed_type = "v_hotellist"
            parsed_content = {"city":city_id}
    elif ugc_type == "hotel_country" :
        if ugc_content.isdigit() :
            country_id = int(ugc_content)
            parsed_type = "v_hotellist"
            parsed_content = {"country":country_id}
    elif ugc_type == "hotel_detail" :
        if ugc_content.isdigit() :
            hotel_id = int(ugc_content)
            parsed_type = "v_hotel"
            parsed_content = hotel_id
    elif ugc_type == "goto_hotel" :
        if ugc_content.isdigit() :
            hotel_id = int(ugc_content)
            parsed_type = "v_goto_hotel"
            parsed_content = hotel_id
    elif parsedUrl.netloc == "appview.qyer.com" :
        if parsedUrl.path.startswith("/bbs") :
            tid = 0
            m = re.search(r"thread-(\d+)-", parsedUrl.path)
            if m : 
                tid = int(m.group(1))
            else :
                m = re.search(r"tid=(\d+)", parsedUrl.query)
                if m : tid = int(m.group(1))
            if tid : 
                parsed_type = "v_tbbs"
                parsed_content = tid
    elif parsedUrl.netloc == "plan.qyer.com" or (parsedUrl.netloc == "appview.qyer.com" and parsedUrl.path.startswith("/plan")) :
        if "/calendar" in parsedUrl.path or "/cost" in parsedUrl.path or "/message" in parsedUrl.path or "/map" in parsedUrl.path or "/traffic" in parsedUrl.path or "/hotel" in parsedUrl.path :
            m = re.search(r"/(\d+)/", parsedUrl.path)
            if m : 
                plan_id = int(m.group(1))
                parsed_type = "v_plan"
                parsed_content = plan_id
        else :
            m = re.search(r"tripapp/(\d+)", parsedUrl.path)
            if m :
                plan_id = int(m.group(1))
                parsed_type = "v_plan"
                parsed_content = plan_id
            else :
                m = re.search(r"poi_(\d+)", parsedUrl.path)
                if m :
                    poi_id = int(m.group(1))
                    parsed_type = "v_place"
                    parsed_content = {"poi":poi_id}
    else :
        parse_rowSet = urlparser.parse_url(path)
        if parse_rowSet["action_type"] == "view_guide_detail" : 
            if parse_rowSet["action_content"]["guide_id"] : 
                parsed_type = "v_guide"
                parsed_content = parse_rowSet["action_content"]["guide_id"]
        elif parse_rowSet["action_type"] == "view_lm_list" :
            country_id = parse_rowSet["action_content"]["country_id"]
            city_id = parse_rowSet["action_content"]["city_id"]
            parsed_type = "v_lmlist"
            parsed_content={}
            if country_id :
                parsed_type = "v_lmlist"
                parsed_content["country"] = country_id
            if city_id : 
                parsed_type = "v_lmlist"
                parsed_content["city"] = city_id
        elif parse_rowSet["action_type"].startswith("search") :
            parsed_type = "search"
            parsed_content={}
            parsed_content["wd"] = parse_rowSet["action_content"].get("wd", "")
            parsed_content["wd_code"] = parse_rowSet["action_content"].get("wd_code", "")
            parsed_content["cityids"] = parse_rowSet["action_content"].get("cityids", [])
            parsed_content["countryids"] = parse_rowSet["action_content"].get("countryids", [])
        elif parse_rowSet["action_type"] == "goto_hotel" :
            if "hotel_id" in parse_rowSet["action_content"] : 
                parsed_type = "v_goto_hotel"
                parsed_content=  parse_rowSet["action_content"]["hotel_id"]
        
    if parsed_type and parsed_content:
        print '%s\t%s\t%s' % (dora_id, parsed_type, parsed_content)

for line in sys.stdin:
    rowSet = line.strip().split('\t')
    if (len(rowSet) < 5):
        continue
    try:
        extract_web_action(rowSet)
    except Exception as ex:
        print '%s\t%s\t%s' % (rowSet[0], rowSet[1], ex)
mongo_db.close()
     
if __name__ == '__main__':
    pass